function f_check_mdl_lock()
{   #仅支持5.1, 5.5, 5.6, 5.7 8.x的版本
    if [ "$(grep -cP "^(5\.(1|5|6|7)|8)" <<< "${mysql_version}")x" == "0x" ]
    then
        f_logging "$(eval echo ${log_addr}):REMAIND" "当前版本 '${mysql_version}' 不检查mdl状态" "1" "0" "0"
        return 0
    fi
    declare fun_name
    fun_name="f_check_mdl_lock"
    check_ip_list="${1}"
    check_role="${2}"
    check_db="${3}"
    check_table="$(sed 's/,//g' <<< "${4}"|sed 's/ //g')"
    check_pid="${5}"
    check_slave_count="${6}"
    check_tools="${7}"

    if [ "${check_role}x" == "slavex" ]
    then #如果是从库就检查10秒, 10秒以后还有mdl就处理
        check_mdl_timeout=10
    fi

    if [ "${check_tools}x" == "gh-ostx" ]
    then
        mark_table="_${check_table}_ghc"
    elif [ "${check_tools}x" == "pt-online-schema-changex" ]
    then
        mark_table="_${check_table}_new"
    elif [ "$(grep -cP "^(alter|drop|rename|truncate)$" <<< "${check_tools}")x" == "1x" ]
    then
        mark_table="_${check_table}_is_mark_table"
    else
        f_logging "$(eval echo ${log_addr}):ERROR" "不识别 '${check_tools}', 这是一个异常错误." "2" "1" "0" "0"
        return 1
    fi

    stop_mark="1" #mdl检查工作的停止标志变量, 0表示可以停止检查, 即结束while循环
    [ "${first_check_mdl_mark}x" == "0x" ] && sleep 10      #表示先sleep 10秒后再开始检查mdl
    [ "${first_check_mdl_mark}x" == "1x" ] && sleep 10      #表示先sleep 10秒后再开始检查mdl
    [ -z "${check_mark}" ] && check_mark=1  #打印 提示 检查mdl信息, 1表示打印, 0表示不打印
    [ -z "${print_mark}" ] && print_mark=1  #打印 mdl信息, 1表示打印, 0表示不打印
    [ "${first_check_mdl_mark}x" == "2x" ] && check_mark=0 && print_mark=0 #first_check_mdl_mark=2表示不是第一次检查mdl就不打印信息了
    [ "${first_check_mdl_mark}x" == "3x" ] && check_mark=0 && print_mark=0 #first_check_mdl_mark=3表示不是第一次检查mdl就不打印信息了
    check_start_time="$(date +%s)" #记录检查开始时间, 这是对出现100%以后做判断, 如果出现100%以后, 半小时还没完成是有问题的
    check_start_time_0="${check_start_time}"    #记录一个时间, 这是对半小时没有出现0%, 或者半小时后还一直是0%, 做判断, 这是有问题的
    check_cur_time="1"             #初始化记录检查当前时间差
    start_time_opt="0"             #记录检查开始时间是否可改, 0表示可改, 1表示不可改
    while :   #检查mdl
    do
        if [ -f "${check_mdl_mark_file}" ]
        then
            date +%s > ${check_mdl_mark_file} #每秒去更新一下这个文件, 表示这个任务还在工作
        else #不存在这个文件就终止检查, 这个文件可能被f_check_daemonize函数删除
            stop_mark=2 && break  
        fi

        if [ "${check_role}x" == "masterx" ] && [ "${check_mark}x" == "1x" ]
        then #如果是master, 就检查连接数
            [ ! -f "${check_connect_file}" ] && touch ${check_connect_file}
            [ ! -f "${check_thread_file}" ] && touch ${check_thread_file}
            f_check_connect "${check_mdl_mark_file}" "100" "${check_tools}" "${check_db}" "${check_table}" "${mysql_user}" "connect" "连接数" &
            f_check_connect "${check_mdl_mark_file}" "100" "${check_tools}" "${check_db}" "${check_table}" "${mysql_user}" "thread" "活跃连接数" &
        fi

        f_update_status_for_task "${check_tools}" "${mysql_host}" "${check_table}"

        slave_is_ok=0 #所有从库是否都没有mdl
        osc_is_run=0  #osc工具运行的标志
        [ -f "${osc_run_mark_file_list[0]}" -o -f "${osc_run_mark_file_list[1]}" ] && osc_is_run=1 || osc_is_run=0
        if [ "${osc_is_run}x" == "1x" ]
        then #如果文件还在就检查一下进程是否真的存在, 有可能进程已经异常退出

            osc_is_run="$(ps -ef|grep -w "${check_db}"|grep -w "${check_table}"|grep -w "${mysql_port}"|grep -wi "${check_tools}"|grep -wc "${mysql_host}")" || true
            [ "${osc_is_run}x" != "0x" ] && osc_is_run=1
        fi
        [ "${kill_master_for_null_query}x" == "2x" ] && osc_is_run=1 #多检查一次

        for check_ip in ${check_ip_list}
        do
            if [ "${check_mark}x" == "1x" ]
            then #提示信息只显示一次
                f_logging "$(eval echo ${log_addr}):REMAIND" "${check_role} '${check_ip}:${mysql_port}' 开始检查mdl状态" "1" "0" "0"
            fi

            #检查是否存在标志表, 用于判断osc在主库是否已经执行完毕, 如果存在则表示还在执行
            get_mark_table_sql="select count(*) from ${isdbt} where table_name = '${mark_table}' and table_schema = '${check_db}'"

            f_print_progress_info "${check_tools}" "${check_table}" || true

            if [ "${osc_is_run}x" == "1x" ] && [ "${check_tools}x" == "gh-ostx" ]
            then #检查mdl时间超过30min了, 且进程还存在, 则取判断一下标志表是否存在, 如果不存在, 则可能触发了gh-ost的bug, 创建一个标志文件, 给守护进程
                master_mark_table="$(f_exe_sql_in_mysql "${get_mark_table_sql}" "" "" "" "$(eval echo ${log_addr})")" #其实就是_new _gho _ghc这种表

                _is_copy="0"       #判断当前改表日志是否存在copy数据的记录, 如果有记录则表示改表在进行了
                _is_copy_0="0"     #判断当前改表日志是否存在copy 0.0%数据的记录
                _is_copy100="0"    #判断当前改表日志是否存在copy 100.0%数据的记录(10x.0%)

                if [ -f "${res_log}" ]
                then
                    cat ${res_log} >> ${res_log}.back && {
                        _is_copy="$(grep -Pc "Copy: [0-9]*/[0-9]* [0-9.]*%; Applied: [0-9]*; Backlog:" ${res_log}.back)"
                        _is_copy0="$(grep -Pc "Copy: 0/[0-9]* 0\.0%; Applied: 0; Backlog:" ${res_log}.back)"
                        _is_copy100="$(grep -Pc "Copy: [0-9]*/[0-9]* 10[0-9]\.[0-9]%; Applied: [0-9]*; Backlog:" ${res_log}.back)"
                        rm -f "${res_log}.back"
                    } || true
                fi

                check_cur_time="$(($(date +%s)-${check_start_time_0}))" #记录检查当前时间差, 判断是否没有0%, 及半小时一直是0.0%这种日志

                if [ "${_is_copy100}x" != "0x" ] && [ "${start_time_opt}x" == "0x" ] && [ "${master_mark_table}x" == "0x" ] #开始出现100%且标志表名不存在
                then #如果存在或超过100%的, 如果有就判断时间, 最多允许半小时结束, 就是说如果出现了100%的记录, 半小时内没完成就kill（标志表不存在情况下）
                    check_start_time="$(date +%s)" && start_time_opt="1" && sleep 1
                elif [ "${_is_copy}x" != "0x" ] && [ "${_is_copy0}x" != "${_is_copy}x" ] && [ "${start_time_opt}x" == "0x" ]
                then #每次更新时间, 因为这种情况视为正常
                    check_start_time="$(date +%s)" && sleep 1
                elif [ "${check_cur_time}" -gt "${ghost_wait_time}" ] && [ "${_is_copy}x" == "0x" -o "${_is_copy0}x" == "${_is_copy}x" ]
                then #只要这个时间小于1800秒就认为正常, 就是说允许ghost 半小时没任何反应, 这个可能是bug
                    echo 0 > "${bug_mark_file}" && return 0 #半小时了都还没出现0.0%或者一直是0.0%这种日志视为失败
                elif [ "${start_time_opt}x" == "0x" ] #其他情况都是正常的
                then
                    check_start_time="$(date +%s)" && sleep 1
                fi

                check_cur_time="$(($(date +%s)-${check_start_time}))" #出现100%后, 记录检查当前时间差
                if [ "${master_mark_table}x" == "0x" ] && [ "${check_cur_time}" -gt "${ghost_wait_time}" ] #1800s
                then #如果主库的标志表不存在, 且check_cur_time超过阈值, 则表示改表任务没开始且不返回状态, 有可能是遇到bug
                     #如果存在或超过100%的, 如果有就判断时间, 最多允许半小时结束, 就是说如果出现了100%的记录, 半小时内没完成就kill（标志表不存在情况下）
                    echo 1 > "${bug_mark_file}" && return 0
                fi
            fi

            if [ "${osc_is_run}x" == "0x" ]
            then #如果osc执行完了, 就检查master的标志表是否还存在, 如果还存在则表示osc被异常中断
                sleep 0.2
                master_mark_table="$(f_exe_sql_in_mysql "${get_mark_table_sql}" "" "" "" "$(eval echo ${log_addr})")" #其实就是_new _gho _ghc这种表
                f_logging "$(eval echo ${log_addr}):REMAIND" "mdl检查子线程退出, 原因是'${check_ip}:${mysql_port}' '${check_role}' 主线程已经退出(${osc_is_run}) ps -ef|grep -w ${check_db}|grep -w ${check_table}|grep -w ${mysql_port}|grep -wi ${check_tools}|grep -wc ${mysql_host}" "1" "0" "0" || true
            fi

            if [ "${osc_is_run}x" == "0x" ] && [ "${check_role}x" == "masterx" ] && [ "${master_mark_table}x" != "0x" ]
            then #这种情况可能不会存在
                f_logging "$(eval echo ${log_addr}):REMAIND" "任务异常 '${check_ip}:${mysql_port}' '${check_role}' 检查mdl状态退出, 主线程已经退出, 但是改表标志文件还存在" "1" "0" "0"
                stop_mark=0 && break
            elif [ "${osc_is_run}x" == "0x" ] && [ "${check_role}x" == "masterx" ]
            then #任务正常退出的场景
                f_logging "$(eval echo ${log_addr}):REMAIND" "任务结束 '${check_ip}:${mysql_port}' '${check_role}' 检查mdl状态退出, 主线程已经退出." "1" "0" "0"
                stop_mark=0 && break
            elif [ "${osc_is_run}x" == "0x" ] && [ "${master_mark_table}x" != "0x" ]
            then #这种情况, 改表进程异常中断了
                f_logging "$(eval echo ${log_addr}):REMAIND" "任务异常 '${check_ip}:${mysql_port}' '${check_role}' 检查mdl状态退出, 主线程已经退出,  但是改表标志文件还存在2." "1" "0" "0"
                stop_mark=0 && break
            else #检查slave
                slave_mark_table="$(f_exe_sql_in_mysql "${get_mark_table_sql}" "${check_ip}" "${mysql_port}" "slave" "$(eval echo ${log_addr})")"
            fi

            if [ "${osc_is_run}x" == "0x" ] && [ "${master_mark_table}x" == "0x" ] && [ "${slave_mark_table}x" == "0x" ]
            then
                slave_is_ok="$((${slave_is_ok}+1))"
                if [ "${slave_is_ok}x" == "${check_slave_count}x" ]
                then
                    f_logging "$(eval echo ${log_addr}):REMAIND" "所有slave都没有检查到mdl状态, '${check_role}' 检查退出" "1" "0" "0"
                    stop_mark=0 && break
                fi
            fi
             
            #检查mdl, master:30秒, slave:10秒, gh-ost:30s, pt-osc:10s
            select_concat_sql="select concat('kill ',id,';') from ${isdbp}"
            select_sql="select * from ${isdbp}"
            select_count_sql="select count(*) from ${isdbp}"
            #如果是master, 则mdl状态一定是改表用户, 如果是从库可能是因为system user, 因为是复制线程
            select_where_info="where STATE like 'Waiting for table metadata lock%' and user in ('${mysql_user}','system user') and db = '${check_db}';"
            select_mdl_sql="${select_count_sql} ${select_where_info}"
            is_has_mdl="$(f_exe_sql_in_mysql "${select_mdl_sql}" "${check_ip}" "${mysql_port}" "slave" "$(eval echo ${log_addr})")"

            for ((wait_mdl_time=${wait_mdl_time}; wait_mdl_time<${check_mdl_timeout}; wait_mdl_time++))
            do  #默认是10秒, gh-ost是30
                [ "${is_has_mdl}x" == "0x" ] && break
                sleep 1
                is_has_mdl="$(f_exe_sql_in_mysql "${select_mdl_sql}" "${check_ip}" "${mysql_port}" "slave" "$(eval echo ${log_addr})")"
            done

            if [ "${is_has_mdl}x" != "0x" ] && [ "${print_mark}x" == "1x" ]
            then
                #将mdl状态入库并发邮件
                { f_check_mdl_and_delay "${mysql_host}" "${mysql_port}" "master" "${user_order_id}" && f_get_html_info_prepare_for_mdl "${mysql_host}" "master" "${user_order_id}" "${mysql_port}";} || true
                print_mark=0
                select_mdl_sql="use ${isdb};select * from PROCESSLIST where STATE like 'Waiting for table metadata lock%';"
                f_logging "$(eval echo ${log_addr}):REMAIND" "'${check_ip}:${mysql_port}' mdl等待信息如下" "1" "0" "0"
                f_exe_sql_in_mysql "${select_mdl_sql}" "${check_ip}" "${mysql_port}" "slave" "$(eval echo ${log_addr})"|awk -F'\t' '{
                    print "[ ID : "$1" ] [ USER : "$2" ] [ IP : "$3" ] [ DB : "$4" ] [ COMMAND : "$5" ] [ TIME : "$6" ] [ STATE : "$7" ] [ SQL : "$8" ]"}'
                f_logging "$(eval echo ${log_addr}):REMAIND" "'${check_ip}:${mysql_port}' mdl等待信息如上" "1" "0" "0"
                select_mdl_sql="use ${isdb};select trx_mysql_thread_id, trx_state, trx_started, trx_query from INNODB_TRX ;"
                f_logging "$(eval echo ${log_addr}):REMAIND" "'${check_ip}:${mysql_port}' mdl可能的阻塞源信息如下" "1" "0" "0"
                f_exe_sql_in_mysql "${select_mdl_sql}" "${check_ip}" "${mysql_port}" "slave" "$(eval echo ${log_addr})"|{ grep -w ${check_table}|awk -F'\t' '{
                    print "[ ID : "$1" ] [ STATE : "$2" ] [ START TIME : "$3" ] [ SQL : "$4" ]"}' || true;}
                f_logging "$(eval echo ${log_addr}):REMAIND" "'${check_ip}:${mysql_port}' mdl可能的阻塞源信息如上" "1" "0" "0"
            fi

            if [ "${is_has_mdl}x" != "0x" ]
            then
                if [ "${kill_master_for_null_query}x" == "0x" ]
                then #先kill空连接, 如果不行就再kill select, 再不行: master就kill自身, slave就退出mdl检查
                    kill_master_for_null_query=1
                    f_kill_mysql_query "${check_ip}" "${mysql_port}" "null" "${check_role}" "mdl" "${check_tools}" "${check_db}" "${check_table}" || true
                    continue
                elif [ "${kill_master_for_null_query}x" == "1x" ]
                then #先kill空连接, 如果不行就再kill select, 再不行: master就kill自身, slave就退出mdl检查
                    kill_master_for_null_query=2
                    f_kill_mysql_query "${check_ip}" "${mysql_port}" "select" "${check_role}" "mdl" "${check_tools}" "${check_db}" "${check_table}" || true
                elif [ "${check_role}x" == "slavex" ]
                then
                    f_logging "$(eval echo ${log_addr}):REMAIND" "'${check_ip}:${mysql_port}' 存在mdl等待, 但是没有找到可以释放的连接, 现在退出该节点mdl检查" "1" "0" "0"
                    check_ip_list="$(sed "s/\<${check_ip}\>//g" <<< "${check_ip_list}")" #将该节点剔除检查mdl任务
                    continue
                fi
                kill_master_for_null_query=3
            fi

            if [ "${is_has_mdl}x" != "0x" ] && [ "${check_role}x" == "masterx" ]
            then #如果是master, 则mdl状态一定是改表用户, 如果是从库可能是因为system user, 因为是复制线程
                if [ "${first_check_mdl_mark}x" == "0x" ] || [ "${first_check_mdl_mark}x" == "3x" ]
                then
                    select_where_info="where state like 'Waiting for table metadata lock%' and user = '${mysql_user}' and db = '${check_db}';"
                else
                    return 5 #return 5表示第一次检查到mdl返回, 主要是给暂停任务使用, 第一次检查到mdl就先暂停15min
                fi
 
                if [ ! -f "${has_mdl_lock_file}" ]
                then #如果存在mdl锁就生成标志文件
                    touch ${has_mdl_lock_file} || true
                fi

                if [ ! -f "${pause_file_name}" ]
                then
                    touch ${pause_file_name} || return 2
                fi
                f_logging "$(eval echo ${log_addr}):ERROR" "master节点 '${check_ip}:${mysql_port}' kill掉空连接后还存在mdl锁等待, 现在将终止改表任务" "1" "0" "0"
                kill_sql="${select_concat_sql} ${select_where_info}"
                get_kill_sql="$(f_exe_sql_in_mysql "${kill_sql}" "${check_ip}" "${mysql_port}" "slave" "$(eval echo ${log_addr})")" && exe_state="${?}" || exe_state="${?}"
                if [ ${exe_state} -eq 0 ] && [ -n "${get_kill_sql}" ]
                then #kill掉mysql层的改表进程, 这里是打印改表进程的信息
                    f_logging "$(eval echo ${log_addr}):REMAIND" "现在将在 '${check_ip}:${mysql_port}' kill掉如下连接" "1" "0" "0"
                    kill_sql="${select_sql} ${select_where_info}"
                    f_exe_sql_in_mysql "${kill_sql}" "${check_ip}" "${mysql_port}" "slave" "$(eval echo ${log_addr})"|awk -F'\t' '{
                        print "[ ID : "$1" ] [ USER : "$2" ] [ IP : "$3" ] [ DB : "$4" ] [ COMMAND : "$5" ] [ TIME : "$6" ] [ STATE : "$7" ] [ SQL : "$8" ]"}'
                    f_logging "$(eval echo ${log_addr}):REMAIND" "现在将在 '${check_ip}:${mysql_port}' kill掉如上连接" "1" "0" "0"
                elif [ ${exe_state} -eq 0 ] && [ -z "${get_kill_sql}" ]
                then
                    f_logging "$(eval echo ${log_addr}):REMAIND" "'${check_ip}:${mysql_port}' 在mysql没有找到改表连接" "1" "0" "0"
                else
                    f_logging "$(eval echo ${log_addr}):WARN" "'${check_ip}:${mysql_port}' 查找改表连接遇到异常" "2" "0" "0"
                fi

                f_kill_osc_process "${check_tools}" "${check_ip}" "${mysql_port}" "${check_db}" "${check_table}" "${mysql_user}" || true
                [ "$(grep -cP "^(alter|drop|rename|truncate)$" <<< "${check_tools}")x" == "1x" ] && return "${exe_state}"
                
                #清理临时表
                f_delete_tmp_table "${check_tools}" "${check_db}" "${check_table}" "${check_ip}" "原因是遇到mdl导致改表任务终止" || true
                return 0
            fi
        done
        check_mark=0
        [ "${stop_mark}x" == "0x" ] && return 0
        sleep 0.2
    done
    return 0
}

